'''Removal of stop words'''
import nltk
import re


def removePunc( text ):
	punc = [".", ",",":","(",")", "]", "[", "/", "'", '"', '<', '>', '!', '-', '&', '#', ';', '%', '_', '^', '+', '$', '@', '*']
	for item in punc:
		text = text.replace(item, ' ');
	for i in range(10):
		text = text.replace(str(i)+"th", ' ');
	text = text.replace( '1st' , ' ');
	text = text.replace( '2nd' , ' ');
	text = text.replace( '3rd' , ' ');
	return text;

def removeStopWords( text ):
	stopwords = open('stopwords.txt','r').read().split()
		
	text = removePunc( text );
	text = text.split(' ')
	i = 1
	new_text = ''
	for word in text:
		if word.lower() not in stopwords and word != '': 
			word = word.strip('\n')
			if word.isdigit() is False and len(word) > 2:
				if i > 20:
					new_text += word + ' '
				i += 1
	return new_text


'''f = open('filteredText.txt', 'r')
allText = f.read()
text = removeStopWords( allText )
exit()
url = "outputText.txt"
f = open(url, 'w')
print text
exit()
f.write(text)'''


